From 893e6246123e7f47c927a5bbc027898e72392770 Mon Sep 17 00:00:00 2001 From: Woochanlee Date: Thu, 4 Aug 2022 19:18:35 +0900 Subject: [PATCH] Introduce Intelligent Ui Automation thorough Screen Analyzer +libaurum: Change name ElementType -> Type +libaurum: efl_util API resource protection with mutex +ScreenAnalyzer: Add mqtt_enabled option for build with or without mosquitto lib ScreenAnalyzer Enable build command : gbs build -A armv7l --define "mqtt 1" --include-all +ScreenAnlayzer: Add SetServerIP for user can choose server ScreenAnalyzer: Code re-arrange and fix warning ScreenAnalyzer: Add commands for GSA with specific features (FR15, FR13, FR14, FR8) ScreenAnalyzer: Add command to get application toolkit info (FR3, FR1) ScreenAnalyzer: Use secvideo_capture API instead of efl_util's for performance (NFR2) ScreenAnalyzer: Add IObject interface then implement in Node, UiObject, SaObject (FR10, FR6, FR11) ScreenAnalyzer: Gets focused app Id and check the atspi window activated or not (FR10) ScreenAnalyzer: Support Ui-Analzyer (FR12) ScreenAnalyzer: Support compatibility Atspi with Screen Analyzer in command (FR9) ScreenAnalyzer: Add functions for request ScreenAnalyzer (FR4) ScreenAnalyzer: Gets UI information from Screen Analyzer and parse it to saobject (FR4, FR5, FR7, FR8) ScreenAnalyzer: Check the extenal application state at runtime (FR3) ScreenAnalyzer: Introduce object class for screen analayzer (FR5) Change-Id: I4f75f61f93985b6ad9a1fe172ddff0882697ad30 --- libaurum/inc/Accessibility/AccessibleNode.h | 111 +++-- libaurum/inc/Aurum.h | 5 + .../Impl/Accessibility/AtspiAccessibleNode.h | 5 + .../inc/Impl/Accessibility/AtspiWrapper.h | 1 + .../Impl/Accessibility/MockAccessibleNode.h | 6 + libaurum/inc/Impl/TizenDeviceImpl.h | 2 + libaurum/inc/Interface/IObject.h | 84 ++++ libaurum/inc/Misc/Rect.h | 16 + libaurum/inc/PartialMatch.h | 6 +- libaurum/inc/SaObject.h | 105 +++++ .../ScreenAnalyzer/ScreenAnalyzerWatcher.h | 147 +++++++ libaurum/inc/UiDevice.h | 64 +++ libaurum/inc/UiObject.h | 171 +++++--- libaurum/inc/UiSelector.h | 13 + libaurum/meson.build | 21 + .../Accessibility/AccessibleApplication.cc | 4 +- libaurum/src/Accessibility/AccessibleNode.cc | 22 +- .../Impl/Accessibility/AtspiAccessibleNode.cc | 17 + .../src/Impl/Accessibility/AtspiWrapper.cc | 8 +- .../Impl/Accessibility/MockAccessibleNode.cc | 4 + libaurum/src/Impl/TizenDeviceImpl.cc | 9 +- libaurum/src/PartialMatch.cc | 34 +- libaurum/src/SaObject.cc | 96 +++++ .../ScreenAnalyzer/ScreenAnalyzerWatcher.cc | 386 ++++++++++++++++++ libaurum/src/ScreenAnalyzer/meson.build | 3 + libaurum/src/UiDevice.cc | 45 ++ libaurum/src/UiObject.cc | 23 +- libaurum/src/UiSelector.cc | 15 +- libaurum/src/meson.build | 9 + meson.build | 4 + meson_options.txt | 6 + .../inc/AurumServiceImpl.h | 6 + .../inc/Commands/Commands.h | 2 + .../Commands/EnableScreenAnalyzerCommand.h | 34 ++ .../Commands/GetActiveAppToolkitNameCommand.h | 34 ++ org.tizen.aurum-bootstrap/meson.build | 2 + .../src/AurumServiceImpl.cc | 17 + .../src/Commands/DumpObjectTreeCommand.cc | 74 +++- .../Commands/EnableScreenAnalyzerCommand.cc | 41 ++ .../src/Commands/FindElementCommand.cc | 170 +++++--- .../src/Commands/FindElementsCommand.cc | 178 +++++--- .../GetActiveAppToolkitNameCommand.cc | 33 ++ .../src/Commands/GetSizeCommand.cc | 79 +++- packaging/aurum.spec | 16 +- protocol/aurum.proto | 75 ++-- tests/Test_UiObject.cc | 4 +- tests/Test_UiSelector.cc | 6 +- 47 files changed, 1911 insertions(+), 302 deletions(-) create mode 100644 libaurum/inc/Interface/IObject.h create mode 100644 libaurum/inc/SaObject.h create mode 100644 libaurum/inc/ScreenAnalyzer/ScreenAnalyzerWatcher.h create mode 100644 libaurum/src/SaObject.cc create mode 100644 libaurum/src/ScreenAnalyzer/ScreenAnalyzerWatcher.cc create mode 100644 libaurum/src/ScreenAnalyzer/meson.build create mode 100644 org.tizen.aurum-bootstrap/inc/Commands/EnableScreenAnalyzerCommand.h create mode 100644 org.tizen.aurum-bootstrap/inc/Commands/GetActiveAppToolkitNameCommand.h create mode 100644 org.tizen.aurum-bootstrap/src/Commands/EnableScreenAnalyzerCommand.cc create mode 100644 org.tizen.aurum-bootstrap/src/Commands/GetActiveAppToolkitNameCommand.cc diff --git a/libaurum/inc/Accessibility/AccessibleNode.h b/libaurum/inc/Accessibility/AccessibleNode.h index 5b9179a..2fa3a41 100644 --- a/libaurum/inc/Accessibility/AccessibleNode.h +++ b/libaurum/inc/Accessibility/AccessibleNode.h @@ -25,6 +25,7 @@ #include #include "IEventConsumer.h" +#include "IObject.h" #include "Rect.h" #include "config.h" @@ -83,7 +84,7 @@ enum class NodeFeatureProperties { * * @since_tizen 6.5 */ -class AccessibleNode : public std::enable_shared_from_this, public IEventConsumer { +class AccessibleNode : public std::enable_shared_from_this, public IEventConsumer, public IObject { public: /** * @brief AccessibleNode constructor. @@ -148,6 +149,52 @@ public: */ void invalidate(); +public: + /** + * @copydoc UiObject::getId() + */ + std::string getId() const override; + + /** + * @copydoc UiObject::getType() + */ + std::string getType() const override; + + /** + * @copydoc UiObject::getScreenBoundingBox() + */ + const Rect getScreenBoundingBox() const override; + + /** + * @copydoc UiObject::getOcrText() + */ + std::string getOcrText() const override; + + /** + * @copydoc UiObject::isFocusable() + */ + bool isFocusable() const override; + + /** + * @copydoc UiObject::isFocused() + */ + bool isFocused() const override; + + /** + * @copydoc UiObject::isClickable() + */ + bool isClickable() const override; + + /** + * @copydoc UiObject::isActive() + */ + bool isActive() const override; + + /** + * @copydoc UiObject::isShowing() + */ + bool isShowing() const override; + public: /** * @copydoc UiObject::getText() @@ -155,14 +202,19 @@ public: std::string getText() const; /** - * @copydoc UiObject::getPkg() + * @copydoc UiObject::setText() */ - std::string getPkg() const; + bool setText(std::string text); /** - * @copydoc UiObject::getId() + * @copydoc UiObject::setOcrText() */ - std::string getId() const; + void setOcrText(std::string text); + + /** + * @copydoc UiObject::getPkg() + */ + std::string getPkg() const; /** * @copydoc UiObject::getAutomationId() @@ -174,30 +226,25 @@ public: */ std::string getRole() const; - /** - * @copydoc UiObject::getType() - */ - std::string getType() const; - /** * @copydoc UiObject::getStyle() */ std::string getStyle() const; /** - * @copydoc UiObject::getXPath() + * @copydoc UiObject::getToolkitName() */ - std::string getXPath() const; + std::string getToolkitName() const; /** - * @copydoc UiObject::getScreenBoundingBox() + * @copydoc UiObject::getWindowBoundingBox() */ - Rect getScreenBoundingBox() const; + const Rect getWindowBoundingBox() const; /** - * @copydoc UiObject::getWindowBoundingBox() + * @copydoc UiObject::getXPath() */ - Rect getWindowBoundingBox() const; + std::string getXPath() const; /** * @copydoc UiObject::getMinValue() @@ -239,26 +286,11 @@ public: */ bool isChecked() const; - /** - * @copydoc UiObject::isClickable() - */ - bool isClickable() const; - /** * @copydoc UiObject::isEnabled() */ bool isEnabled() const; - /** - * @copydoc UiObject::isFocusable() - */ - bool isFocusable() const; - - /** - * @copydoc UiObject::isFocused() - */ - bool isFocused() const; - /** * @copydoc UiObject::isLongClickable() */ @@ -284,16 +316,6 @@ public: */ bool isVisible() const; - /** - * @copydoc UiObject::isShowing() - */ - bool isShowing() const; - - /** - * @copydoc UiObject::isActive() - */ - bool isActive() const; - public: /** * @brief Print Node information. @@ -363,6 +385,11 @@ public: */ virtual void updateXPath() = 0; + /** + * @copydoc UiObject::updateToolkitName() + */ + virtual void updateToolkitName() = 0; + /** * @copydoc UiObject::updateValue() */ @@ -478,6 +505,7 @@ public: protected: std::string mText; + std::string mOcrText; std::string mPkg; std::string mRole; std::string mId; @@ -485,6 +513,7 @@ protected: std::string mType; std::string mStyle; std::string mXPath; + std::string mToolkitName; Rect mScreenBoundingBox; Rect mWindowBoundingBox; int mSupportingIfaces; diff --git a/libaurum/inc/Aurum.h b/libaurum/inc/Aurum.h index 0003c39..c916371 100644 --- a/libaurum/inc/Aurum.h +++ b/libaurum/inc/Aurum.h @@ -53,6 +53,7 @@ #include "Waiter.h" #include "ISearchable.h" #include "IDevice.h" +#include "IObject.h" #include "Accessible.h" #include "Point2D.h" #include "Size2D.h" @@ -61,5 +62,9 @@ #include "Comparer.h" #include "A11yEvent.h" #include "Runnables.h" +#ifdef MQTT_ENABLED +#include "ScreenAnalyzerWatcher.h" +#include "SaObject.h" +#endif #endif diff --git a/libaurum/inc/Impl/Accessibility/AtspiAccessibleNode.h b/libaurum/inc/Impl/Accessibility/AtspiAccessibleNode.h index 5b8a3b6..ffd1816 100644 --- a/libaurum/inc/Impl/Accessibility/AtspiAccessibleNode.h +++ b/libaurum/inc/Impl/Accessibility/AtspiAccessibleNode.h @@ -121,6 +121,11 @@ public: */ void updateXPath() override; + /** + * @copydoc UiObject::updateToolkitName() + */ + void updateToolkitName() override; + /** * @copydoc UiObject::updateValue() */ diff --git a/libaurum/inc/Impl/Accessibility/AtspiWrapper.h b/libaurum/inc/Impl/Accessibility/AtspiWrapper.h index f8f69b1..d34ceba 100644 --- a/libaurum/inc/Impl/Accessibility/AtspiWrapper.h +++ b/libaurum/inc/Impl/Accessibility/AtspiWrapper.h @@ -67,6 +67,7 @@ public: static gboolean Atspi_value_set_current_value(AtspiValue *iface, gdouble value, GError **error); static gdouble Atspi_value_get_minimum_increment(AtspiValue *iface, GError **error); static guint Atspi_accessible_get_process_id(AtspiAccessible *node, GError **error); + static gchar *Atspi_accessible_get_toolkit_name(AtspiAccessible *node, GError **error); private: static std::recursive_mutex mMutex; diff --git a/libaurum/inc/Impl/Accessibility/MockAccessibleNode.h b/libaurum/inc/Impl/Accessibility/MockAccessibleNode.h index 68ef294..2a8afa6 100644 --- a/libaurum/inc/Impl/Accessibility/MockAccessibleNode.h +++ b/libaurum/inc/Impl/Accessibility/MockAccessibleNode.h @@ -123,6 +123,12 @@ public: */ void updateXPath() override; + /** + * @brief TBD + * @since_tizen 7.0 + */ + void updateToolkitName() override; + /** * @brief TBD * @since_tizen 7.0 diff --git a/libaurum/inc/Impl/TizenDeviceImpl.h b/libaurum/inc/Impl/TizenDeviceImpl.h index 8852a45..253685b 100644 --- a/libaurum/inc/Impl/TizenDeviceImpl.h +++ b/libaurum/inc/Impl/TizenDeviceImpl.h @@ -23,6 +23,7 @@ #include #include +#include using namespace Aurum; @@ -200,6 +201,7 @@ private: * @brief TBD */ Size2D mScreenSize; + static std::mutex CaptureMutex; }; } diff --git a/libaurum/inc/Interface/IObject.h b/libaurum/inc/Interface/IObject.h new file mode 100644 index 0000000..0e6ee9f --- /dev/null +++ b/libaurum/inc/Interface/IObject.h @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#ifndef _IOBJECT_H_ +#define _IOBJECT_H_ + +#include "config.h" +#include "Rect.h" +#include + +namespace Aurum { + + +class IObject { +public: + + /** + * @brief IObject destructor + */ + virtual ~IObject() {} + + /** + * @copydoc UiObject::getId() + */ + virtual std::string getId() const = 0; + + /** + * @copydoc UiObject::getType() + */ + virtual std::string getType() const = 0; + + /** + * @copydoc UiObject::getScreenBoundingBox() + */ + virtual const Rect getScreenBoundingBox() const = 0; + + /** + * @copydoc UiObject::getOcrText() + */ + virtual std::string getOcrText() const = 0; + + /** + * @copydoc UiObject::isFocusable() + */ + virtual bool isFocusable() const = 0; + + /** + * @copydoc UiObject::isFocused() + */ + virtual bool isFocused() const = 0; + + /** + * @copydoc UiObject::isClickable() + */ + virtual bool isClickable() const = 0; + + /** + * @copydoc UiObject::isActive() + */ + virtual bool isActive() const = 0; + + /** + * @copydoc UiObject::isShowing() + */ + virtual bool isShowing() const = 0; +}; + +} + +#endif diff --git a/libaurum/inc/Misc/Rect.h b/libaurum/inc/Misc/Rect.h index 70410af..cf9323d 100644 --- a/libaurum/inc/Misc/Rect.h +++ b/libaurum/inc/Misc/Rect.h @@ -112,6 +112,22 @@ public: */ T height() const { return mBottomRight.y - mTopLeft.y; } + /** + * @brief Checks whether the given point in the rect or not. + * + * @return true if point located in rect, otherwise false + * + * @since_tizen 7.0 + */ + bool isInRect(const Point2D &point) const + { + if (point.x >= mTopLeft.x && point.x <= mBottomRight.x + && point.y >= mTopLeft.y && point.y <= mBottomRight.y) + return true; + + return false; + } + /** * @brief Checks given Rect is same as this or not. * diff --git a/libaurum/inc/PartialMatch.h b/libaurum/inc/PartialMatch.h index 8413543..4806683 100644 --- a/libaurum/inc/PartialMatch.h +++ b/libaurum/inc/PartialMatch.h @@ -129,13 +129,15 @@ private: * * @param[in] selector @UiSelector * @param[in] node @AccessibleNode target node + * @param[in] device @UiDevice * - * @return ture if satisfied, else false + * @return true if satisfied, else false * * @since_tizen 6.5 */ static bool checkCriteria(const std::shared_ptr selector, - const std::shared_ptr node); + const std::shared_ptr node, + std::shared_ptr device); /** * @brief Checks text matched or not. diff --git a/libaurum/inc/SaObject.h b/libaurum/inc/SaObject.h new file mode 100644 index 0000000..422f264 --- /dev/null +++ b/libaurum/inc/SaObject.h @@ -0,0 +1,105 @@ +/* + * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#ifndef _SA_OBJECT_H_ +#define _SA_OBJECT_H_ + +#include "config.h" +#include + +namespace Aurum { + +/** + * @class SaObject + * + * @ingroup aurum + * + * @brief A SaObject is a result of Screen Analyzer. + */ +class SaObject : public IObject, public std::enable_shared_from_this { +public: + + /** + * @brief SaObject constructor with initial values. + * + * @since_tizen 7.0 + */ + SaObject(std::string id, std::string type, Rect geometry, std::string ocrText, std::vector states); + + /** + * @brief SaObject destructor. + * + * @since_tizen 7.0 + */ + ~SaObject(); + + /** + * @copydoc UiObject::getId() + */ + std::string getId() const override; + + /** + * @copydoc UiObject::getType() + */ + std::string getType() const override; + + /** + * @copydoc UiObject::getScreenBoundingBox() + */ + const Rect getScreenBoundingBox() const override; + + std::string getOcrText() const override; + + /** + * @copydoc UiObject::isFocusable() + */ + bool isFocusable() const override; + + /** + * @copydoc UiObject::isFocused() + */ + bool isFocused() const override; + + /** + * @copydoc UiObject::isClickable() + */ + bool isClickable() const override; + + /** + * @copydoc UiObject::isActive() + */ + bool isActive() const override; + + /** + * @copydoc UiObject::isShowing() + */ + bool isShowing() const override; + +private: + std::string mId; + std::string mElementType; + Rect mScreenBoundingBox; + std::string mOcrText; + bool mIsFocusable; + bool mIsFocused; + bool mIsClickable; + bool mIsActive; +}; + +} + +#endif diff --git a/libaurum/inc/ScreenAnalyzer/ScreenAnalyzerWatcher.h b/libaurum/inc/ScreenAnalyzer/ScreenAnalyzerWatcher.h new file mode 100644 index 0000000..d013c8a --- /dev/null +++ b/libaurum/inc/ScreenAnalyzer/ScreenAnalyzerWatcher.h @@ -0,0 +1,147 @@ +/* + * Copyright (c) 2021 Samsung Electronics Co., Ltd All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#ifndef _SCREEN_ANALYZER_WATCHER_H_ +#define _SCREEN_ANALYZER_WATCHER_H_ + +#include + +namespace Aurum { + +/** + * @class ScreenAnalyzerWatcher + * + * @ingroup aurum + * + * @brief ScreenAnalyzerWatcher communicates with Screen Analyzer Server via MQTT Protocol(mosquitto). + * It requests screen analyzer to server and parses the result that can be used in aurum. + */ +class ScreenAnalyzerWatcher { +public: + + /** + * @brief ScreenAnalyzerWatcher constructor. + * + * @since_tizen 7.0 + */ + ScreenAnalyzerWatcher(); + + /** + * @brief ScreenAnalyzerWatcher destructor. + * + * @since_tizen 7.0 + */ + ~ScreenAnalyzerWatcher(); + + /** + * @brief When message(analyze result) is received from the broker. + * parse the data and create SaObject. + * + * @param[in] mosq mosquitto instance + * @param[in] obj user data provided in mosquitto_new + * @param[in] msg the message data + * + * @since_tizen 7.0 + */ + static void onMessage(struct mosquitto *mosq, void *obj, const struct mosquitto_message *msg); + + /** + * @brief Publish current screen image to the broker. + * capture current screen and publish a message on a given topic. + * + * @since_tizen 7.0 + */ + void PublishData(); + + /** + * @brief Publish current screen image to the broker. + * here capture current screen and publish a message on a given topic. + * + * @since_tizen 7.0 + */ + std::vector> GetSaObjects(); + + /** + * @copydoc PartialMatch::checkCriteria() + */ + bool checkCriteria(const std::string textA, const std::string textB, const bool textPartialMatch); + + /** + * @copydoc PartialMatch::checkCriteria() + */ + bool checkCriteria(const bool boolA, const bool boolB); + + /** + * @copydoc PartialMatch::checkCriteria() + */ + bool checkCriteria(const std::shared_ptr selector, const std::shared_ptr node); + + /** + * @brief find object from screen analyzer result. + * + * @param[in] selector @UiSelector + + * + * @return SaObject vector if found, else nulltpr + * + * @since_tizen 7.0 + */ + std::vector> findSaObjects(const std::shared_ptr selector); + + /** + * @brief find object from screen analyzer result. + * + * @param[in] selector @UiSelector + + * + * @return SaObject if found, else nulltpr + * + * @since_tizen 7.0 + */ + std::shared_ptr findSaObject(const std::shared_ptr selector); + + /** + * @brief Get focused app name. + * + * @param[in] selector @UiSelector + * + * @return SaObject if found, else nulltpr + * + * @since_tizen 7.0 + */ + std::string GetFocusedAppId(); + + /** + * @brief Set server IP address. + * + * @param[in] ip std::string + * + * @since_tizen 7.0 + */ + void SetServerIp(std::string ip); + +private: + struct mosquitto *mMosq; + static std::string mPkgName; + static std::vector> mSaObjects; + static bool mLoadDone; + std::string mIp; +}; + +} + +#endif diff --git a/libaurum/inc/UiDevice.h b/libaurum/inc/UiDevice.h index ba21d4a..05f7c01 100644 --- a/libaurum/inc/UiDevice.h +++ b/libaurum/inc/UiDevice.h @@ -36,6 +36,11 @@ #include #include +#ifdef MQTT_ENABLED +#include "SaObject.h" +#include "ScreenAnalyzerWatcher.h" +#endif + namespace Aurum { /** @@ -367,6 +372,7 @@ public: * @brief Gets currently enabled applications root window. * * @return AccessibleNode ptr vector + * * @since_tizen 6.5 */ std::vector> getWindowRoot() const; @@ -375,10 +381,64 @@ public: * @brief Gets currently window information from window system. * * @return TizenWindow ptr vector + * * @since_tizen 7.0 */ std::vector> getTizenWindowInfo() const; +#ifdef MQTT_ENABLED + /** + * @brief Gets screen analyzer Object vector. + * + * @return SaObject ptr vector + * + * @since_tizen 7.0 + */ + std::vector> getSaObject(); + + /** + * @brief Gets ScreenAnalyzerWatcher instance. + * + * @retrun ScreenAnalyzerWatcher ptr + * + * @since_tizen 7.0 + */ + std::shared_ptr getSAWatcher(); +#endif + /** + * @brief Requests current screen analyze. + * + * @since_tizen 7.0 + */ + void RequestScreenAnalyze(); + + /** + * @brief Gets external application launched or not. + * + * @return true external application launched, otherwise false + * + * @since_tizen 7.0 + */ + bool getExternalAppLaunched(); + + /** + * @brief Sets whether to use screen analyzer for Tizen apps. + * + * @param[in] withScreenAnalyzer boolean value + * + * @since_tizen 7.0 + */ + void setWithScreenAnalyzer(bool withScreenAnalyzer); + + /** + * @brief Gets whether to use screen analyzer for Tizen apps. + * + * @return true screen analyzer use for Tizen apps, otherwise false + * + * @since_tizen 7.0 + */ + bool getWithScreenAnalyzer(); + private: /** * @brief Waits process idle. @@ -414,6 +474,10 @@ private: IDevice *mDeviceImpl; const Waiter *mWaiter; static std::vector> mTizenWindows; + bool mIsWithSA; +#ifdef MQTT_ENABLED + static std::shared_ptr mSAWatcher; +#endif }; } diff --git a/libaurum/inc/UiObject.h b/libaurum/inc/UiObject.h index 018d1ad..1863560 100644 --- a/libaurum/inc/UiObject.h +++ b/libaurum/inc/UiObject.h @@ -22,6 +22,7 @@ #include "Accessible.h" #include "ISearchable.h" +#include "IObject.h" #include "UiSelector.h" #include "Waiter.h" @@ -54,7 +55,7 @@ public: * such as object's properties, states, geometry information. * also user can send and receive event via this class. */ -class UiObject : public ISearchable , public std::enable_shared_from_this { +class UiObject : public ISearchable , public std::enable_shared_from_this, public IObject { public: /** * @brief UiObject constructor with device, selector, node pointer. @@ -155,6 +156,88 @@ public: */ bool waitFor(const std::function condition) const; +public: + /** + * @brief Gets object's Id. + * + * @return string + * + * @since_tizen 6.5 + */ + std::string getId() const override; + + /** + * @brief Gets object's type. + * + * @return string + * + * @since_tizen 7.0 + */ + std::string getType() const override; + + /** + * @brief Gets object's geometry of the screen. + * + * @return @Rect + * + * @since_tizen 6.5 + */ + const Rect getScreenBoundingBox() const override; + + /** + * @brief Gets object's ocr text. + * + * @return string + * + * @since_tizen 7.0 + */ + std::string getOcrText() const override; + + /** + * @brief Gets object's focusable property. + * + * @return true if focusable else false + * + * @since_tizen 6.5 + */ + bool isFocusable() const override; + + /** + * @brief Gets object's focused property. + * + * @return true if focused else false + * + * @since_tizen 6.5 + */ + bool isFocused() const override; + + /** + * @brief Gets object's clickable property. + * + * @return true if clickable else false + * + * @since_tizen 6.5 + */ + bool isClickable() const override; + + /** + * @brief Gets object's active property. + * + * @return true if active else false + * + * @since_tizen 6.5 + */ + bool isActive() const override; + + /** + * @brief Gets object's showing property. + * + * @return true if showing else false + * + * @since_tizen 6.5 + */ + bool isShowing() const override; + public: /** * @brief Gets object's parent. @@ -210,15 +293,6 @@ public: */ std::string getApplicationPackage() const; - /** - * @brief Gets object's Id. - * - * @return string - * - * @since_tizen 6.5 - */ - std::string getId() const; - /** * @brief Gets object's automation Id. * @@ -228,15 +302,6 @@ public: */ std::string getAutomationId() const; - /** - * @brief Gets object's type. - * - * @return string - * - * @since_tizen 6.5 - */ - std::string getElementType() const; - /** * @brief Gets object's style. * @@ -264,6 +329,15 @@ public: */ std::string getRole() const; + /** + * @brief Gets object's Toolkit name. + * + * @return string + * + * @since_tizen 7.0 + */ + std::string getToolkitName() const; + /** * @brief Gets object's XPath. * @@ -285,13 +359,13 @@ public: bool setText(std::string text); /** - * @brief Gets object's geometry of the screen. + * @brief Sets object's ocr text. * - * @return @Rect + * @param[in] text string * - * @since_tizen 6.5 + * @since_tizen 7.0 */ - const Rect getScreenBoundingBox() const; + void setOcrText(std::string text); /** * @brief Gets object's geometry of the window. @@ -376,15 +450,6 @@ public: */ bool isChecked() const; - /** - * @brief Gets object's clickable property. - * - * @return true if clickable else false - * - * @since_tizen 6.5 - */ - bool isClickable() const; - /** * @brief Gets object's enabled property. (to get object enabled. disabled state) * @@ -394,24 +459,6 @@ public: */ bool isEnabled() const; - /** - * @brief Gets object's focusable property. - * - * @return true if focusable else false - * - * @since_tizen 6.5 - */ - bool isFocusable() const; - - /** - * @brief Gets object's focused property. - * - * @return true if focused else false - * - * @since_tizen 6.5 - */ - bool isFocused() const; - /** * @brief Gets object's longclickable property. * @@ -457,24 +504,6 @@ public: */ bool isVisible() const; - /** - * @brief Gets object's showing property. - * - * @return true if showing else false - * - * @since_tizen 6.5 - */ - bool isShowing() const; - - /** - * @brief Gets object's active property. - * - * @return true if active else false - * - * @since_tizen 6.5 - */ - bool isActive() const; - /** * @brief Performs a click action on object. * @@ -566,6 +595,12 @@ public: */ void updatePid() const; + /** + * @brief Updates object's toolkit name information from atspi server. + * + * @since_tizen 7.0 + */ + void updateToolkitName() const; /** * @brief Sets focus to object. * diff --git a/libaurum/inc/UiSelector.h b/libaurum/inc/UiSelector.h index c8f510d..98f544d 100644 --- a/libaurum/inc/UiSelector.h +++ b/libaurum/inc/UiSelector.h @@ -92,6 +92,17 @@ public: */ UiSelector *text(std::string text); + /** + * @brief Sets the search criteria to match the object's ocr text. + * + * @param[in] text object text + * + * @return UiSelector class instance + * + * @since_tizen 7.0 + */ + UiSelector *ocrText(std::string text); + /** * @brief Sets the search criteria to match the object's text has given text. * @@ -367,6 +378,7 @@ public: std::string mStyle; std::string mTextPartialMatch; std::string mXPath; + std::string mOcrText; bool mMatchId; bool mMatchAutomationId; @@ -377,6 +389,7 @@ public: bool mMatchStyle; bool mMatchTextPartialMatch; bool mMatchXPath; + bool mMatchOcrText; bool mMatchChecked; bool mMatchCheckable; diff --git a/libaurum/meson.build b/libaurum/meson.build index 66c87a0..7ecdcc5 100644 --- a/libaurum/meson.build +++ b/libaurum/meson.build @@ -33,6 +33,13 @@ libaurum_install_inc = [ './inc/Aurum.h', ] +if get_option('mqtt_enabled') == true + libaurum_install_inc += [ + './inc/SaObject.h', + './inc/ScreenAnalyzer/ScreenAnalyzerWatcher.h', + ] +endif + libaurum_inc = [ include_directories('./inc'), include_directories('./inc/Accessibility'), @@ -44,6 +51,12 @@ libaurum_inc = [ root_inc, ] +if get_option('mqtt_enabled') == true + libaurum_inc += [ + include_directories('./inc/ScreenAnalyzer'), + ] +endif + libaurum_src = [] subdir('src') @@ -64,6 +77,14 @@ if get_option('tizen') == true ] endif +if get_option('mqtt_enabled') == true + libaurum_dep += [ + dependency('libmosquitto'), + dependency('jsoncpp'), + dependency('capi-appfw-app-manager'), + ] +endif + libaurum_lib = library('aurum', libaurum_src, dependencies: libaurum_dep, include_directories: libaurum_inc, diff --git a/libaurum/src/Accessibility/AccessibleApplication.cc b/libaurum/src/Accessibility/AccessibleApplication.cc index 21c8fd0..e72c085 100644 --- a/libaurum/src/Accessibility/AccessibleApplication.cc +++ b/libaurum/src/Accessibility/AccessibleApplication.cc @@ -46,8 +46,8 @@ std::vector> AccessibleApplication::getActiveW auto children = getWindows(); children.erase(std::remove_if(children.begin(), children.end(), [](auto child){ - return !(child->isShowing()); // Active, Showing are same meaning currently + return !(child->isActive() && child->isShowing() && (child->getAccessibleNode()->getChildCount() > 0)); }), children.end()); return children; -} \ No newline at end of file +} diff --git a/libaurum/src/Accessibility/AccessibleNode.cc b/libaurum/src/Accessibility/AccessibleNode.cc index 19e81a4..e7bdbd1 100644 --- a/libaurum/src/Accessibility/AccessibleNode.cc +++ b/libaurum/src/Accessibility/AccessibleNode.cc @@ -31,7 +31,7 @@ AccessibleNode::~AccessibleNode() } AccessibleNode::AccessibleNode() -: mText{""}, mPkg{""}, mRole{""}, mId{""}, mAutomationId{""}, mType{""}, mStyle{""}, mXPath{""}, +: mText{""}, mOcrText{""}, mPkg{""}, mRole{""}, mId{""}, mAutomationId{""}, mType{""}, mStyle{""}, mXPath{""}, mToolkitName{""}, mScreenBoundingBox{0,0,0,0}, mWindowBoundingBox{0,0,0,0}, mSupportingIfaces(0), mFeatureProperty(0), mPid(0), mMinValue{0.0}, mMaxValue{0.0}, mValue{0.0}, mIncrement{0.0}, mValid{true}, mLock{} { } @@ -44,6 +44,7 @@ std::string AccessibleNode::description() { ss << "\"mAutomationId\":\"" << this->mAutomationId << "\", "; ss << "\"mRole\":\"" << this->mRole << "\", "; ss << "\"mText\":\"" << this->mText << "\", "; + ss << "\"mOcrText\":\"" << this->mOcrText << "\", "; ss << "\"mPkg\":\"" << this->mPkg << "\", "; ss << "\"mType\":\"" << this->mType << "\", "; ss << "\"mStyle\":\"" << this->mStyle << "\", "; @@ -120,6 +121,16 @@ std::string AccessibleNode::getText() const return mText; } +std::string AccessibleNode::getOcrText() const +{ + return mOcrText; +} + +void AccessibleNode::setOcrText(std::string text) +{ + mOcrText = text; +} + std::string AccessibleNode::getPkg() const { return mPkg; @@ -155,12 +166,17 @@ std::string AccessibleNode::getXPath() const return mXPath; } -Rect AccessibleNode::getScreenBoundingBox() const +std::string AccessibleNode::getToolkitName() const +{ + return mToolkitName; +} + +const Rect AccessibleNode::getScreenBoundingBox() const { return mScreenBoundingBox; } -Rect AccessibleNode::getWindowBoundingBox() const +const Rect AccessibleNode::getWindowBoundingBox() const { return mWindowBoundingBox; } diff --git a/libaurum/src/Impl/Accessibility/AtspiAccessibleNode.cc b/libaurum/src/Impl/Accessibility/AtspiAccessibleNode.cc index 8d04ab3..4130cd4 100644 --- a/libaurum/src/Impl/Accessibility/AtspiAccessibleNode.cc +++ b/libaurum/src/Impl/Accessibility/AtspiAccessibleNode.cc @@ -144,6 +144,17 @@ void AtspiAccessibleNode::updateName() } } +void AtspiAccessibleNode::updateToolkitName() +{ + AtspiWrapper::Atspi_accessible_clear_cache(mNode); + + gchar *toolkitName = AtspiWrapper::Atspi_accessible_get_toolkit_name(mNode, NULL); + if (toolkitName) { + mToolkitName = toolkitName; + g_free(toolkitName); + } +} + void AtspiAccessibleNode::updateApplication() { AtspiWrapper::Atspi_accessible_clear_cache(mNode); @@ -299,6 +310,12 @@ void AtspiAccessibleNode::refresh(bool updateAll) g_free(name); } + gchar *toolkitName = AtspiWrapper::Atspi_accessible_get_toolkit_name(mNode, NULL); + if (toolkitName) { + mToolkitName = toolkitName; + g_free(toolkitName); + } + AtspiAccessible *app = AtspiWrapper::Atspi_accessible_get_application(mNode, NULL); if (app) { gchar *pkg = AtspiWrapper::Atspi_accessible_get_name(app, NULL); diff --git a/libaurum/src/Impl/Accessibility/AtspiWrapper.cc b/libaurum/src/Impl/Accessibility/AtspiWrapper.cc index 93e1c37..ceef6cf 100644 --- a/libaurum/src/Impl/Accessibility/AtspiWrapper.cc +++ b/libaurum/src/Impl/Accessibility/AtspiWrapper.cc @@ -211,4 +211,10 @@ guint AtspiWrapper::Atspi_accessible_get_process_id(AtspiAccessible *node, GErro { std::unique_lock lock(mMutex); return atspi_accessible_get_process_id(node, error); -} \ No newline at end of file +} + +gchar *AtspiWrapper::Atspi_accessible_get_toolkit_name(AtspiAccessible *node, GError **error) +{ + std::unique_lock lock(mMutex); + return atspi_accessible_get_toolkit_name(node, error); +} diff --git a/libaurum/src/Impl/Accessibility/MockAccessibleNode.cc b/libaurum/src/Impl/Accessibility/MockAccessibleNode.cc index 8900dd8..1650e10 100644 --- a/libaurum/src/Impl/Accessibility/MockAccessibleNode.cc +++ b/libaurum/src/Impl/Accessibility/MockAccessibleNode.cc @@ -121,6 +121,10 @@ void MockAccessibleNode::updatePid() { } +void MockAccessibleNode::updateToolkitName() +{ +} + bool MockAccessibleNode::setFocus() { return false; diff --git a/libaurum/src/Impl/TizenDeviceImpl.cc b/libaurum/src/Impl/TizenDeviceImpl.cc index f8b7d42..ab87c4a 100644 --- a/libaurum/src/Impl/TizenDeviceImpl.cc +++ b/libaurum/src/Impl/TizenDeviceImpl.cc @@ -38,6 +38,9 @@ using namespace AurumInternal; #define NANO_SEC 1000000000.0 #define MICRO_SEC 1000000 + +std::mutex TizenDeviceImpl::CaptureMutex = std::mutex{}; + TizenDeviceImpl::TizenDeviceImpl() : mFakeTouchHandle{0}, mFakeKeyboardHandle{0}, mFakeWheelHandle{0}, tStart{}, isTimerStarted{false}, mTouchSeq{} { @@ -275,6 +278,7 @@ bool TizenDeviceImpl::takeScreenshot(std::string path, float scale, int quality) efl_util_screenshot_h screenshot = NULL; tbm_surface_h tbm_surface = NULL; + CaptureMutex.lock(); screenshot = efl_util_screenshot_initialize(mScreenSize.width, mScreenSize.height); if (screenshot) { @@ -285,13 +289,15 @@ bool TizenDeviceImpl::takeScreenshot(std::string path, float scale, int quality) tbm_surface_destroy(tbm_surface); } else { efl_util_screenshot_deinitialize(screenshot); + CaptureMutex.unlock(); return false; } efl_util_screenshot_deinitialize(screenshot); } else { + CaptureMutex.unlock(); return false; } - + CaptureMutex.unlock(); return true; } @@ -345,6 +351,7 @@ long long TizenDeviceImpl::getSystemTime(TimeRequestType type) const Size2D TizenDeviceImpl::getScreenSize() { TizenDeviceImpl *obj = static_cast(this); + LOGI("getScreenSize : %d %d", obj->mScreenSize.width , obj->mScreenSize.height); return obj->mScreenSize; } diff --git a/libaurum/src/PartialMatch.cc b/libaurum/src/PartialMatch.cc index 54332b4..ba030d1 100644 --- a/libaurum/src/PartialMatch.cc +++ b/libaurum/src/PartialMatch.cc @@ -52,8 +52,37 @@ std::string PartialMatch::debugPrint() } bool PartialMatch::checkCriteria(const std::shared_ptr selector, - const std::shared_ptr node) + const std::shared_ptr node, + std::shared_ptr device) { +#ifdef MQTT_ENABLED + if (selector->mMatchOcrText) + { + if (!device->getWithScreenAnalyzer() || !node->isShowing() || !(node->getText().size() == 0)) return false; + //FIXME: getSaObjectHasText(); + auto saObjs = device->getSAWatcher()->GetSaObjects(); + + node->updateExtents(); + auto objBoundingBox = node->getScreenBoundingBox(); + for (auto saObj : saObjs) { + if (saObj->getOcrText().size() != 0) { + if (objBoundingBox.isInRect(saObj->getScreenBoundingBox().midPoint()) && + ((objBoundingBox.width() * objBoundingBox.height() * 1.5) > (saObj->getScreenBoundingBox().width() * saObj->getScreenBoundingBox().height())) && + ((objBoundingBox.width() * objBoundingBox.height() < (saObj->getScreenBoundingBox().width() * saObj->getScreenBoundingBox().height() * 1.5)))) + { + LOGI("Text Set %s ", saObj->getOcrText().c_str()); + node->setOcrText(saObj->getOcrText()); + } + } + } + + if (node->getOcrText().size() == 0) return false; + if (checkCriteria(selector->mOcrText, node->getOcrText(), 1)) return false; + + LOGI("node ocr = %s, selector ocr = %s",node->getOcrText().c_str(), selector->mOcrText.c_str()); + } +#endif + if (selector->mMatchText || selector->mMatchTextPartialMatch) { node->updateName(); if (selector->mMatchText && checkCriteria(selector->mText, node->getText(), 0)) return false; @@ -115,12 +144,13 @@ std::shared_ptr PartialMatch::accept(const std::shared_ptr mDevice = UiDevice::getInstance(); if ((selector->mMinDepth && relativeDepth < selector->mMinDepth) || (selector->mMaxDepth && relativeDepth > selector->mMaxDepth)) { return std::shared_ptr(nullptr); } - if (PartialMatch::checkCriteria(selector, node)) + if (PartialMatch::checkCriteria(selector, node, mDevice)) match = new PartialMatch(selector, absoluteDepth); return std::shared_ptr(match); } diff --git a/libaurum/src/SaObject.cc b/libaurum/src/SaObject.cc new file mode 100644 index 0000000..ade4f32 --- /dev/null +++ b/libaurum/src/SaObject.cc @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2021 Samsung Electronics Co., Ltd All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "Aurum.h" + +#include +#include +#include +#include +#include +#include + +#include "ScreenAnalyzerWatcher.h" + +using namespace Aurum; + +SaObject::SaObject(std::string id, std::string type, Rect geometry, std::string ocrText, std::vector states) + : mId(id), mElementType(type), mScreenBoundingBox(geometry), mOcrText(ocrText) +{ + LOGI("SaObject Constructor"); + mIsFocusable = false; + mIsFocused = false; + mIsClickable = false; + mIsActive = false; + for (auto state : states) + { + if (state == "clickable") mIsClickable = true; + else if (state == "focused") mIsFocused = true; + else if (state == "focusable") mIsFocusable = true; + else if (state == "active") mIsActive = true; + } +} + +SaObject::~SaObject() +{ + LOGI("SaObject destructor"); +} + +std::string SaObject::getId() const +{ + return mId; +} + +std::string SaObject::getType() const +{ + return mElementType; +} + +const Rect SaObject::getScreenBoundingBox() const +{ + return mScreenBoundingBox; +} + +std::string SaObject::getOcrText() const +{ + return mOcrText; +} + +bool SaObject::isFocusable() const +{ + return mIsFocusable; +} + +bool SaObject::isFocused() const +{ + return mIsFocused; +} + +bool SaObject::isClickable() const +{ + return mIsClickable; +} + +bool SaObject::isActive() const +{ + return mIsActive; +} + +bool SaObject::isShowing() const +{ + return true; +} \ No newline at end of file diff --git a/libaurum/src/ScreenAnalyzer/ScreenAnalyzerWatcher.cc b/libaurum/src/ScreenAnalyzer/ScreenAnalyzerWatcher.cc new file mode 100644 index 0000000..be0a93c --- /dev/null +++ b/libaurum/src/ScreenAnalyzer/ScreenAnalyzerWatcher.cc @@ -0,0 +1,386 @@ +/* + * Copyright (c) 2021 Samsung Electronics Co., Ltd All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "Aurum.h" + +#include "ScreenAnalyzerWatcher.h" +#include +#include +#include +#include "SaObject.h" +#include +#include +#include +#include +#include +#include +#include + +using namespace Aurum; + +//Initialize static member variables +std::vector> ScreenAnalyzerWatcher::mSaObjects; +bool ScreenAnalyzerWatcher::mLoadDone; +std::string ScreenAnalyzerWatcher::mPkgName; + +void on_connect(struct mosquitto *mosq, void *obj, int reason_code) +{ + LOGI("mosquitto on connect"); + + if(reason_code != 0){ + LOGE("mosquitto on connect error : %d", reason_code); + mosquitto_disconnect(mosq); + } + int rc; + rc = mosquitto_subscribe(mosq, NULL, "screen_analyzer/json_aurum", 1); + if(rc != MOSQ_ERR_SUCCESS){ + LOGE("server on connect subscribe fail"); + mosquitto_disconnect(mosq); + } + +} + +void on_publish(struct mosquitto *mosq, void *obj, int mid) +{ + LOGI("mosquitto on publish"); +} + +void on_subscribe(struct mosquitto *mosq, void *obj, int mid, int qos_count, const int *granted_qos) +{ + LOGI("server on subscribe qos : %d", qos_count); +} + +ScreenAnalyzerWatcher::ScreenAnalyzerWatcher() +{ + LOGI("Screen Analyzer constructor"); + + mosquitto_lib_init(); + LOGI("mosquitto lib init"); + mMosq = mosquitto_new(NULL, true, NULL); + if(mMosq == NULL){ + LOGE("mosquitto new fail"); + return; + } + + mosquitto_connect_callback_set(mMosq, on_connect); + mosquitto_publish_callback_set(mMosq, on_publish); + mosquitto_subscribe_callback_set(mMosq, on_subscribe); + mosquitto_message_callback_set(mMosq, ScreenAnalyzerWatcher::onMessage); + + //FIXME : When the class is created, there is an issue if We do not try to connect the server, + // so I put the AIM team IP to try to connect. + mIp = "10.113.16.21"; //This is AIM default server user can change this using SetServerIp() + + int rc; + rc = mosquitto_connect(mMosq, mIp.c_str(), 1883, 60); + if(rc != MOSQ_ERR_SUCCESS){ + mosquitto_destroy(mMosq); + LOGE("mosquitto connect fail"); + return; + } + rc = mosquitto_loop_start(mMosq); + if(rc != MOSQ_ERR_SUCCESS){ + mosquitto_destroy(mMosq); + LOGE("mosquitto loop fail"); + return; + } +} + +ScreenAnalyzerWatcher::~ScreenAnalyzerWatcher() +{ + LOGI("Screen Analyzer destructor"); + + if (mMosq) + { + mosquitto_disconnect(mMosq); + mosquitto_loop_stop(mMosq, false); + mosquitto_lib_cleanup(); + mMosq = nullptr; + } +} + +void ScreenAnalyzerWatcher::onMessage(struct mosquitto *mosq, void *obj, const struct mosquitto_message *msg) +{ + std::string txt((char *)msg->payload); + + mSaObjects.clear(); + + Json::Reader reader; + Json::Value root; + bool ret = reader.parse(txt, root); + if (!ret) LOGE("mosquitto parse fail"); + + const Json::Value objs = root["objects"]; + LOGI("objects count = %d", root["objectCount"].asInt()); + + std::string id; + std::string type; + Rect geometry; + std::string ocrText; + std::vector states{}; + + std::shared_ptr mDevice = UiDevice::getInstance(); + //Add dummy for window + id = "9999"; + type = "window"; + geometry = {0, 0, mDevice->getScreenSize().width, mDevice->getScreenSize().height}; + states.push_back("active"); + ocrText = mPkgName; + mSaObjects.push_back(std::make_shared(id, type, geometry, ocrText, states)); + + for (unsigned int idx = 0; idx < objs.size(); ++idx) + { + states.clear(); + id = objs[idx]["id"].asString(); + LOGI("objs[%d] ID : %s", idx,id.c_str()); + + type = objs[idx]["type"].asString(); + LOGI("objs[%d] TYPE : %s", idx, objs[idx]["type"].asString().c_str()); + + const Json::Value geo = objs[idx]["geometry"]; + geometry = { geo[0].asInt(), geo[1].asInt(), geo[2].asInt(), geo[3].asInt() }; + LOGI("objs[%d] GEOMETRY : %d %d %d %d", idx, geo[0].asInt(), geo[1].asInt(), geo[2].asInt(), geo[3].asInt()); + + ocrText = objs[idx]["ocrText"].asString(); + LOGI("objs[%d] OCRTEXT : %s", idx, objs[idx]["ocrText"].asString().c_str()); + + const Json::Value sta = objs[idx]["states"]; + for (unsigned int sidx = 0; sidx < sta.size(); ++sidx) { + LOGI("objs[%d] STATE: %s", idx, sta[sidx].asString().c_str()); + states.push_back(sta[sidx].asString()); + } + + mSaObjects.push_back(std::make_shared(id, type, geometry, ocrText, states)); + } + + mLoadDone = true; +} + +void ScreenAnalyzerWatcher::PublishData() +{ + LOGI("Prepare capture data for data publish"); + mLoadDone = false; + void *ptr = NULL; + unsigned char *src = NULL; + unsigned char *dst = NULL; + + efl_util_screenshot_h screenshot = NULL; + tbm_surface_h tbm_surface = NULL; + tbm_surface_info_s info; + std::shared_ptr mDevice = UiDevice::getInstance(); + const int WIDTH = mDevice->getScreenSize().width; + const int HEIGHT = mDevice->getScreenSize().height; + + screenshot = efl_util_screenshot_initialize(WIDTH, HEIGHT); + + if (screenshot) + { + tbm_surface = efl_util_screenshot_take_tbm_surface(screenshot); + if (tbm_surface) + { + tbm_surface_map(tbm_surface, TBM_SURF_OPTION_READ, &info); + + ptr = malloc( WIDTH * HEIGHT * 4 ); + src = (unsigned char *)info.planes[0].ptr; + dst = (unsigned char *)ptr; + + memcpy(dst, src, WIDTH * HEIGHT * 4); + } + else + { + efl_util_screenshot_deinitialize(screenshot); + return; + } + + int payloadlen = WIDTH * HEIGHT * 4; + + LOGI("mosquitto publish start"); + int rc = mosquitto_publish(mMosq, NULL, "screen_analyzer/image_aurum", payloadlen , ptr, 2, false); + if(rc != MOSQ_ERR_SUCCESS) { + LOGE("mosquitto publish fail"); + } + + tbm_surface_destroy(tbm_surface); + efl_util_screenshot_deinitialize(screenshot); + } + else + { + LOGE("Screen shot fail"); + return; + } + + while(!mLoadDone) { + std::this_thread::sleep_for(std::chrono::milliseconds{10}); + } + + if (tbm_surface) { + free(ptr); + tbm_surface_unmap(tbm_surface); + tbm_surface_destroy(tbm_surface); + } +} + +std::vector> ScreenAnalyzerWatcher::GetSaObjects() +{ + return mSaObjects; +} + +bool ScreenAnalyzerWatcher::checkCriteria(const std::string textA, const std::string textB, const bool textPartialMatch) +{ + if (textB.empty()) return true; + + bool rst; + if (textPartialMatch) { + if (textB.find(textA) != std::string::npos) rst = false; + else rst = true; + } + else { + if (!textA.compare(textB)) rst = false; + else rst = true; + } + + return rst; +} + +bool ScreenAnalyzerWatcher::checkCriteria(const bool boolA, const bool boolB) +{ + return boolA != boolB; +} + +bool ScreenAnalyzerWatcher::checkCriteria(const std::shared_ptr selector, + const std::shared_ptr node) +{ + if (selector->mMatchText || selector->mMatchTextPartialMatch) { + if (selector->mMatchText && checkCriteria(selector->mText, node->getOcrText(), 0)) return false; + if (selector->mMatchTextPartialMatch && checkCriteria(selector->mTextPartialMatch, node->getOcrText(), 1)) return false; + } + if (selector->mMatchId) { + if (checkCriteria(selector->mId, node->getId(), 0)) return false; + } + if (selector->mMatchType) { + if (selector->mMatchType && checkCriteria(selector->mType, node->getType(), 0)) return false; + } + if (selector->mMatchClickable && checkCriteria(selector->mIsclickable, node->isClickable())) return false; + if (selector->mMatchFocused && checkCriteria(selector->mIsfocused, node->isFocused())) return false; + if (selector->mMatchFocusable && checkCriteria(selector->mIsfocusable, node->isFocusable())) return false; + if (selector->mMatchActive && checkCriteria(selector->mIsactive, node->isActive())) return false; + if (selector->mMatchShowing && checkCriteria(selector->mIsshowing, node->isShowing())) return false; + + return true; +} + +std::vector> ScreenAnalyzerWatcher::findSaObjects(const std::shared_ptr selector) +{ + std::vector> rets{}; + + LOGI("FindSaObject start"); + for (auto saObj : mSaObjects) { + if (checkCriteria(selector, saObj)) { + LOGI("saObj(%s) pushed", saObj->getId().c_str()); + rets.push_back(saObj); + } + } + + return rets; +} + +std::shared_ptr ScreenAnalyzerWatcher::findSaObject(const std::shared_ptr selector) +{ + LOGI("FindSaObject start"); + for (auto saObj : mSaObjects) { + if (checkCriteria(selector, saObj)) { + LOGI("saObj(%s) pushed", saObj->getId().c_str()); + return saObj; + } + } + + return nullptr; +} + +std::string ScreenAnalyzerWatcher::GetFocusedAppId() +{ + int return_val = APP_MANAGER_ERROR_NONE; + + app_context_h app_context; + char *app_id = NULL; + + return_val = app_manager_get_focused_app_context(&app_context); + if (return_val != APP_MANAGER_ERROR_NONE) { + LOGE("failed to get app-context"); + return NULL; + } + + return_val = app_context_get_app_id(app_context, &app_id); + if (return_val != APP_MANAGER_ERROR_NONE) { + LOGE("failed to get app_id"); + app_context_destroy(app_context); + return NULL; + } + + LOGI("focused app_id = %s", app_id); + + return_val = app_context_destroy(app_context); + if (return_val != APP_MANAGER_ERROR_NONE) { + LOGE("failed to destroy app-context"); + return NULL; + } + + app_context = NULL; + + return app_id; +} + +void ScreenAnalyzerWatcher::SetServerIp(std::string ip) +{ + LOGI("SA Server Initialize"); + mIp = ip; + + if (mMosq) + { + mosquitto_disconnect(mMosq); + mosquitto_loop_stop(mMosq, false); + mosquitto_lib_cleanup(); + mMosq = nullptr; + } + + mosquitto_lib_init(); + LOGI("mosquitto lib init"); + mMosq = mosquitto_new(NULL, true, NULL); + if(mMosq == NULL){ + LOGE("mosquitto new fail"); + return; + } + + mosquitto_connect_callback_set(mMosq, on_connect); + mosquitto_publish_callback_set(mMosq, on_publish); + mosquitto_subscribe_callback_set(mMosq, on_subscribe); + mosquitto_message_callback_set(mMosq, ScreenAnalyzerWatcher::onMessage); + + int rc; + rc = mosquitto_connect(mMosq, mIp.c_str(), 1883, 60); + if (rc != MOSQ_ERR_SUCCESS){ + mosquitto_destroy(mMosq); + LOGE("mosquitto connect fail"); + return; + } + rc = mosquitto_loop_start(mMosq); + if (rc != MOSQ_ERR_SUCCESS){ + mosquitto_destroy(mMosq); + LOGE("mosquitto loop fail"); + return; + } +} diff --git a/libaurum/src/ScreenAnalyzer/meson.build b/libaurum/src/ScreenAnalyzer/meson.build new file mode 100644 index 0000000..5047b43 --- /dev/null +++ b/libaurum/src/ScreenAnalyzer/meson.build @@ -0,0 +1,3 @@ +libaurum_src += [ + files('ScreenAnalyzerWatcher.cc') +] diff --git a/libaurum/src/UiDevice.cc b/libaurum/src/UiDevice.cc index ea76073..6b66377 100644 --- a/libaurum/src/UiDevice.cc +++ b/libaurum/src/UiDevice.cc @@ -42,11 +42,21 @@ using namespace AurumInternal; std::vector> UiDevice::mTizenWindows; static GDBusConnection *system_conn; +#ifdef MQTT_ENABLED +std::shared_ptr UiDevice::mSAWatcher; +#endif + UiDevice::UiDevice() : UiDevice(nullptr) {} UiDevice::UiDevice(IDevice *impl) : mDeviceImpl(impl), mWaiter(new Waiter{this}) { + LOGI("UiDevice constructor"); +#ifdef MQTT_ENABLED + mSAWatcher = std::make_shared(); + mIsWithSA = false; +#endif + LOGI("UiDevice constructor finish"); } UiDevice::~UiDevice() @@ -420,3 +430,38 @@ const Size2D UiDevice::getScreenSize() { return mDeviceImpl->getScreenSize(); } + +#ifdef MQTT_ENABLED +std::vector> UiDevice::getSaObject() +{ + return mSAWatcher->GetSaObjects(); +} + +std::shared_ptr UiDevice::getSAWatcher() +{ + return mSAWatcher; +} +#endif + +void UiDevice::RequestScreenAnalyze() +{ +#ifdef MQTT_ENABLED + mSAWatcher->PublishData(); +#endif +} + +bool UiDevice::getExternalAppLaunched() +{ + auto ret = this->getWindowRoot(); + return (ret.size() > 0) ? false : true; +} + +void UiDevice::setWithScreenAnalyzer(bool withScreenAnalyzer) +{ + mIsWithSA = withScreenAnalyzer; +} + +bool UiDevice::getWithScreenAnalyzer() +{ + return mIsWithSA; +} diff --git a/libaurum/src/UiObject.cc b/libaurum/src/UiObject.cc index fe54f4f..c975859 100644 --- a/libaurum/src/UiObject.cc +++ b/libaurum/src/UiObject.cc @@ -174,7 +174,7 @@ std::string UiObject::getAutomationId() const return getAccessibleNode()->getAutomationId(); } -std::string UiObject::getElementType() const +std::string UiObject::getType() const { return getAccessibleNode()->getType(); } @@ -229,6 +229,22 @@ bool UiObject::setText(std::string text) return getAccessibleNode()->setValue(text); } +std::string UiObject::getOcrText() const +{ + return getAccessibleNode()->getOcrText(); +} + +std::string UiObject::getToolkitName() const +{ + getAccessibleNode()->updateToolkitName(); + return getAccessibleNode()->getToolkitName(); +} + +void UiObject::setOcrText(std::string text) +{ + getAccessibleNode()->setOcrText(text); +} + bool UiObject::isCheckable() const { return getAccessibleNode()->isCheckable(); @@ -349,6 +365,11 @@ void UiObject::updatePid() const mNode->updatePid(); } +void UiObject::updateToolkitName() const +{ + mNode->updateToolkitName(); +} + bool UiObject::setFocus() const { return mNode->setFocus(); diff --git a/libaurum/src/UiSelector.cc b/libaurum/src/UiSelector.cc index 7d703dc..8157b18 100644 --- a/libaurum/src/UiSelector.cc +++ b/libaurum/src/UiSelector.cc @@ -22,10 +22,10 @@ using namespace Aurum; UiSelector::UiSelector() -: mId{}, mAutomationId{}, mRole{}, mText{}, mPkg{}, mType{}, mStyle{}, mTextPartialMatch{}, mXPath{}, +: mId{}, mAutomationId{}, mRole{}, mText{}, mPkg{}, mType{}, mStyle{}, mTextPartialMatch{}, mXPath{}, mOcrText{}, mMatchId{}, mMatchAutomationId{}, mMatchRole{}, mMatchText{}, mMatchPkg{}, mMatchType{}, mMatchStyle{}, - mMatchTextPartialMatch{}, mMatchXPath{}, mMatchChecked{}, mMatchCheckable{}, mMatchClickable{}, mMatchEnabled{}, mMatchFocused{}, - mMatchFocusable{}, mMatchScrollable{}, mMatchSelected{}, mMatchShowing{}, mMatchActive{}, mMatchVisible{}, + mMatchTextPartialMatch{}, mMatchXPath{}, mMatchOcrText{}, mMatchChecked{}, mMatchCheckable{}, mMatchClickable{}, mMatchEnabled{}, + mMatchFocused{}, mMatchFocusable{}, mMatchScrollable{}, mMatchSelected{}, mMatchShowing{}, mMatchActive{}, mMatchVisible{}, mMatchSelectable{}, mMinDepth{}, mMaxDepth{}, mIschecked{}, mIscheckable{}, mIsclickable{}, mIsenabled{}, mIsfocused{}, mIsfocusable{}, mIsscrollable{}, mIsselected{}, mIsshowing{}, mIsactive{}, mIsvisible{}, mIsselectable{}, mChild{}, mParent{} @@ -40,6 +40,7 @@ std::string UiSelector::description() if(!this->mAutomationId.empty()) ss << "\"mAutomationId\":\"" << this->mAutomationId << "\", "; if(!this->mRole.empty()) ss << "\"mRole\":\"" << this->mRole << "\", "; if(!this->mText.empty()) ss << "\"mText\":\"" << this->mText << "\", "; + if(!this->mOcrText.empty()) ss << "\"mOcrText\":\"" << this->mOcrText << "\", "; if(!this->mTextPartialMatch.empty()) ss << "\"mTextPartialMatch\":\"" << this->mTextPartialMatch << "\", "; if(!this->mXPath.empty()) ss << "\"mXPath\":\"" << this->mXPath << "\", "; if(!this->mPkg.empty()) ss << "\"mPkg\":\"" << this->mPkg << "\", "; @@ -49,6 +50,7 @@ std::string UiSelector::description() if(this->mMatchAutomationId) ss << "\"mMatchAutomationId\":\"" << ((this->mMatchAutomationId)?"true":"false") << "\", "; if(this->mMatchRole) ss << "\"mMatchRole\":\"" << ((this->mMatchRole)?"true":"false") << "\", "; if(this->mMatchText) ss << "\"mMatchText\":\"" << ((this->mMatchText)?"true":"false") << "\", "; + if(this->mMatchOcrText) ss << "\"mMatchOcrText\":\"" << ((this->mMatchOcrText)?"true":"false") << "\", "; if(this->mMatchTextPartialMatch) ss << "\"mMatchTextPartialMatch\":\"" << ((this->mMatchTextPartialMatch)?"true":"false") << "\", "; if(this->mMatchXPath) ss << "\"mMatchXPath\":\"" << ((this->mMatchXPath)?"true":"false") << "\", "; if(this->mMatchPkg) ss << "\"mMatchPkg\":\"" << ((this->mMatchPkg)?"true":"false") << "\", "; @@ -89,6 +91,13 @@ UiSelector *UiSelector::text(std::string text) return this; } +UiSelector *UiSelector::ocrText(std::string text) +{ + this->mOcrText = text; + this->mMatchOcrText = true; + return this; +} + UiSelector *UiSelector::textPartialMatch(std::string text) { this->mTextPartialMatch = text; diff --git a/libaurum/src/meson.build b/libaurum/src/meson.build index 390e805..778c7b0 100644 --- a/libaurum/src/meson.build +++ b/libaurum/src/meson.build @@ -13,6 +13,15 @@ libaurum_src += [ files('TizenWindow.cc'), ] +if get_option('mqtt_enabled') == true + libaurum_src += [ + files('SaObject.cc'), + ] +endif + subdir('Accessibility') +if get_option('mqtt_enabled') == true + subdir('ScreenAnalyzer') +endif subdir('Impl') subdir('Runnable') diff --git a/meson.build b/meson.build index 1022f82..e6af72e 100644 --- a/meson.build +++ b/meson.build @@ -17,6 +17,10 @@ if get_option('tizen_gcov') == true config_h.set10('TIZEN_GCOV', true) endif +if get_option('mqtt_enabled') == true + config_h.set10('MQTT_ENABLED', true) +endif + root_inc = include_directories('./') subdir('pugixml') diff --git a/meson_options.txt b/meson_options.txt index 726ed1d..ec44da5 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -26,3 +26,9 @@ option('enable_documentation', value: false, description: 'enable generating documentation by using doxygen' ) + +option('mqtt_enabled', + type: 'boolean', + value: false, + description: 'enable mqtt protocol(mosquitto) for Screen Analyzer' +) diff --git a/org.tizen.aurum-bootstrap/inc/AurumServiceImpl.h b/org.tizen.aurum-bootstrap/inc/AurumServiceImpl.h index 119e36a..deb020b 100644 --- a/org.tizen.aurum-bootstrap/inc/AurumServiceImpl.h +++ b/org.tizen.aurum-bootstrap/inc/AurumServiceImpl.h @@ -120,6 +120,12 @@ public: ::grpc::Status setFocus(::grpc::ServerContext *context, const ::aurum::ReqSetFocus *request, ::aurum::RspSetFocus *response) override; + ::grpc::Status getActiveAppToolkitName(::grpc::ServerContext *context, + const ::aurum::ReqGetActiveAppToolkitName *request, + ::aurum::RspGetActiveAppToolkitName *response) override; + ::grpc::Status enableScreenAnalyzer(::grpc::ServerContext *context, + const ::aurum::ReqEnableScreenAnalyzer *request, + ::aurum::RspEnableScreenAnalyzer *response) override; }; #endif diff --git a/org.tizen.aurum-bootstrap/inc/Commands/Commands.h b/org.tizen.aurum-bootstrap/inc/Commands/Commands.h index e0badc1..6445640 100644 --- a/org.tizen.aurum-bootstrap/inc/Commands/Commands.h +++ b/org.tizen.aurum-bootstrap/inc/Commands/Commands.h @@ -49,3 +49,5 @@ #include "Commands/GetScreenSizeCommand.h" #include "Commands/ActionAndWaitEventCommand.h" #include "Commands/SetFocusCommand.h" +#include "Commands/GetActiveAppToolkitNameCommand.h" +#include "Commands/EnableScreenAnalyzerCommand.h" diff --git a/org.tizen.aurum-bootstrap/inc/Commands/EnableScreenAnalyzerCommand.h b/org.tizen.aurum-bootstrap/inc/Commands/EnableScreenAnalyzerCommand.h new file mode 100644 index 0000000..700611e --- /dev/null +++ b/org.tizen.aurum-bootstrap/inc/Commands/EnableScreenAnalyzerCommand.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2021 Samsung Electronics Co., Ltd All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include +#include +#include "Commands/Command.h" +#include "ObjectMapper.h" +#include +#include "config.h" + +class EnableScreenAnalyzerCommand : public Command { +private: + const ::aurum::ReqEnableScreenAnalyzer *mRequest; + ::aurum::RspEnableScreenAnalyzer *mResponse; + +public: + EnableScreenAnalyzerCommand(const ::aurum::ReqEnableScreenAnalyzer *request, + ::aurum::RspEnableScreenAnalyzer*response); + ::grpc::Status execute() override; +}; diff --git a/org.tizen.aurum-bootstrap/inc/Commands/GetActiveAppToolkitNameCommand.h b/org.tizen.aurum-bootstrap/inc/Commands/GetActiveAppToolkitNameCommand.h new file mode 100644 index 0000000..e85aba3 --- /dev/null +++ b/org.tizen.aurum-bootstrap/inc/Commands/GetActiveAppToolkitNameCommand.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2022 Samsung Electronics Co., Ltd All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include +#include +#include "Commands/Command.h" +#include "ObjectMapper.h" +#include +#include "config.h" + +class GetActiveAppToolkitNameCommand : public Command { +private: + const ::aurum::ReqGetActiveAppToolkitName *mRequest; + ::aurum::RspGetActiveAppToolkitName *mResponse; + +public: + GetActiveAppToolkitNameCommand(const ::aurum::ReqGetActiveAppToolkitName *request, + ::aurum::RspGetActiveAppToolkitName *response); + ::grpc::Status execute() override; +}; diff --git a/org.tizen.aurum-bootstrap/meson.build b/org.tizen.aurum-bootstrap/meson.build index 0e90268..fb99f18 100644 --- a/org.tizen.aurum-bootstrap/meson.build +++ b/org.tizen.aurum-bootstrap/meson.build @@ -45,6 +45,8 @@ bootstrap_svr_src += [ files('src/Commands/GetScreenSizeCommand.cc'), files('src/Commands/ActionAndWaitEventCommand.cc'), files('src/Commands/SetFocusCommand.cc'), + files('src/Commands/GetActiveAppToolkitNameCommand.cc'), + files('src/Commands/EnableScreenAnalyzerCommand.cc'), ] bootstrap_svr_dep = [ diff --git a/org.tizen.aurum-bootstrap/src/AurumServiceImpl.cc b/org.tizen.aurum-bootstrap/src/AurumServiceImpl.cc index fe62067..00abf4e 100644 --- a/org.tizen.aurum-bootstrap/src/AurumServiceImpl.cc +++ b/org.tizen.aurum-bootstrap/src/AurumServiceImpl.cc @@ -264,3 +264,20 @@ aurumServiceImpl::~aurumServiceImpl() std::unique_ptr cmd = std::make_unique(request, response); return execute(cmd.get(), true); } + +::grpc::Status aurumServiceImpl::getActiveAppToolkitName(::grpc::ServerContext *context, + const ::aurum::ReqGetActiveAppToolkitName *request, + ::aurum::RspGetActiveAppToolkitName *response) +{ + std::unique_ptr cmd = std::make_unique(request, response); + return execute(cmd.get(), true); +} + +::grpc::Status aurumServiceImpl::enableScreenAnalyzer(::grpc::ServerContext *context, + const ::aurum::ReqEnableScreenAnalyzer *request, + ::aurum::RspEnableScreenAnalyzer *response) +{ + std::unique_ptr cmd = std::make_unique(request, response); + return execute(cmd.get(), true); +} + diff --git a/org.tizen.aurum-bootstrap/src/Commands/DumpObjectTreeCommand.cc b/org.tizen.aurum-bootstrap/src/Commands/DumpObjectTreeCommand.cc index dfcda80..4865ada 100644 --- a/org.tizen.aurum-bootstrap/src/Commands/DumpObjectTreeCommand.cc +++ b/org.tizen.aurum-bootstrap/src/Commands/DumpObjectTreeCommand.cc @@ -55,7 +55,7 @@ void DumpObjectTreeCommand::traverse(::aurum::Element *root, std::shared_ptrset_width(windowSize.width()); windowRect->set_height(windowSize.height()); - root->set_widget_type(obj->getElementType()); + root->set_widget_type(obj->getType()); root->set_widget_style(obj->getElementStyle()); root->set_text(obj->getText()); @@ -91,14 +91,70 @@ void DumpObjectTreeCommand::traverse(::aurum::Element *root, std::shared_ptrelementid().c_str()); - if (mRequest->elementid().length()) { - auto obj = mObjMap->getElement(mRequest->elementid()); - if (!obj) return grpc::Status::OK;; - - auto node = obj->getDescendant(); - ::aurum::Element *root = mResponse->add_roots(); - traverse(root, node, 0); + + std::shared_ptr mDevice = UiDevice::getInstance(); +#ifdef MQTT_ENABLED + if (mDevice->getExternalAppLaunched()) + { + mDevice->RequestScreenAnalyze(); + + auto objs = mDevice->getSAWatcher()->GetSaObjects(); + ::aurum::Element *root; + int idx = 0; + for (auto obj : objs) { + if (!idx) { + root = mResponse->add_roots(); + root->set_elementid(obj->getId()); + root->set_widget_type(obj->getType()); + root->set_text(obj->getOcrText()); + root->set_isclickable(obj->isClickable()); + root->set_isfocused(obj->isFocused()); + root->set_isfocusable(obj->isFocusable()); + root->set_isactive(obj->isActive()); + root->set_isshowing(true); + root->set_isvisible(true); + ::aurum::Rect *rect = root->mutable_geometry(); + const Rect &size = obj->getScreenBoundingBox(); + rect->set_x(size.mTopLeft.x); + rect->set_y(size.mTopLeft.y); + rect->set_width(size.width()); + rect->set_height(size.height()); + } + else { + ::aurum::Element *elm = root->add_child(); + elm->set_elementid(obj->getId()); + elm->set_widget_type(obj->getType()); + elm->set_text(obj->getOcrText()); + elm->set_isclickable(obj->isClickable()); + elm->set_isfocused(obj->isFocused()); + elm->set_isfocusable(obj->isFocusable()); + elm->set_isactive(obj->isActive()); + elm->set_isshowing(true); + elm->set_isvisible(true); + + ::aurum::Rect *rect = elm->mutable_geometry(); + const Rect &size = obj->getScreenBoundingBox(); + rect->set_x(size.mTopLeft.x); + rect->set_y(size.mTopLeft.y); + rect->set_width(size.width()); + rect->set_height(size.height()); + } + + idx++; + } + } + else +#endif + { + LOGI("elementid : %s", mRequest->elementid().c_str()); + if (mRequest->elementid().length()) { + auto obj = mObjMap->getElement(mRequest->elementid()); + if (!obj) return grpc::Status::OK;; + + auto node = obj->getDescendant(); + ::aurum::Element *root = mResponse->add_roots(); + traverse(root, node, 0); + } } return grpc::Status::OK; } diff --git a/org.tizen.aurum-bootstrap/src/Commands/EnableScreenAnalyzerCommand.cc b/org.tizen.aurum-bootstrap/src/Commands/EnableScreenAnalyzerCommand.cc new file mode 100644 index 0000000..5caba46 --- /dev/null +++ b/org.tizen.aurum-bootstrap/src/Commands/EnableScreenAnalyzerCommand.cc @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2021 Samsung Electronics Co., Ltd All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "bootstrap.h" +#include "EnableScreenAnalyzerCommand.h" +#include "UiDevice.h" + +EnableScreenAnalyzerCommand::EnableScreenAnalyzerCommand(const ::aurum::ReqEnableScreenAnalyzer *request, + ::aurum::RspEnableScreenAnalyzer *response) + : mRequest{request}, mResponse{response} +{ +} + +::grpc::Status EnableScreenAnalyzerCommand::execute() +{ + LOGI("EnableScreenAnalyzer test --------------- "); + + LOGI("Screen Analzyer enabled (%d) Servier IP : (%s)", mRequest->enable(), mRequest->serverip().c_str()); + std::shared_ptr mDevice = UiDevice::getInstance(); + mDevice->setWithScreenAnalyzer(mRequest->enable()); +#ifdef MQTT_ENABLED + if (mRequest->serverip().size() > 0) + mDevice->getSAWatcher()->SetServerIp(mRequest->serverip()); +#endif + + return grpc::Status::OK; +} diff --git a/org.tizen.aurum-bootstrap/src/Commands/FindElementCommand.cc b/org.tizen.aurum-bootstrap/src/Commands/FindElementCommand.cc index 929119d..ed92078 100644 --- a/org.tizen.aurum-bootstrap/src/Commands/FindElementCommand.cc +++ b/org.tizen.aurum-bootstrap/src/Commands/FindElementCommand.cc @@ -22,6 +22,9 @@ #include "UiSelector.h" #include "Sel.h" #include "ISearchable.h" +#ifdef MQTT_ENABLED +#include "SaObject.h" +#endif FindElementCommand::FindElementCommand(const ::aurum::ReqFindElement *request, ::aurum::RspFindElement *response) @@ -65,6 +68,7 @@ std::shared_ptr FindElementCommand::getSelector(void) if(mRequest->_packagename_case()) sel->pkg(mRequest->packagename()); if(mRequest->_textpartialmatch_case()) sel->textPartialMatch(mRequest->textpartialmatch()); if(mRequest->_xpath_case()) sel->xpath(mRequest->xpath()); + if(mRequest->_ocrtext_case()) sel->ocrText(mRequest->ocrtext()); return sel; } @@ -74,65 +78,113 @@ std::shared_ptr FindElementCommand::getSelector(void) LOGI("findElement --------------- "); auto searchableObj = getSearchableTop(); auto selector = getSelector(); - - auto found = searchableObj->findObject(selector); - - if (found != nullptr) { - UiObject *obj = found.get(); - obj->refresh(); - if (mObjMap->getElement(obj->getId()) == nullptr) - mObjMap->addElement(std::move(found)); - - LOGI("found object : %p elementId:%s", obj, obj->getId().c_str()); - - ::aurum::Element *elm = mResponse->mutable_element(); - elm->set_elementid(obj->getId()); - elm->set_package(obj->getApplicationPackage()); - - ::aurum::Rect *rect = elm->mutable_geometry(); - const Rect &size = obj->getScreenBoundingBox(); - rect->set_x(size.mTopLeft.x); - rect->set_y(size.mTopLeft.y); - rect->set_width(size.width()); - rect->set_height(size.height()); - - ::aurum::Rect *windowRect = elm->mutable_window_relative_geometry(); - const Rect &windowRelativeSize = obj->getWindowBoundingBox(); - windowRect->set_x(windowRelativeSize.mTopLeft.x); - windowRect->set_y(windowRelativeSize.mTopLeft.y); - windowRect->set_width(windowRelativeSize.width()); - windowRect->set_height(windowRelativeSize.height()); - - elm->set_widget_type(obj->getElementType()); - elm->set_widget_style(obj->getElementStyle()); - - elm->set_text(obj->getText()); - elm->set_xpath(obj->getXPath()); - elm->set_automationid(obj->getAutomationId()); - elm->set_package(obj->getApplicationPackage()); - elm->set_role(obj->getRole()); - - elm->set_ischecked(obj->isChecked()); - elm->set_ischeckable(obj->isCheckable()); - elm->set_isclickable(obj->isClickable()); - elm->set_isenabled(obj->isEnabled()); - elm->set_isfocused(obj->isFocused()); - elm->set_isfocusable(obj->isFocusable()); - elm->set_isscrollable(obj->isScrollable()); - elm->set_isselected(obj->isSelected()); - elm->set_isshowing(obj->isShowing()); - elm->set_isactive(obj->isActive()); - elm->set_isvisible(obj->isVisible()); - elm->set_isselectable(obj->isSelectable()); - - elm->set_minvalue(obj->getMinValue()); - elm->set_maxvalue(obj->getMaxValue()); - elm->set_value(obj->getValue()); - elm->set_increment(obj->getIncrement()); - - mResponse->set_status(::aurum::RspStatus::OK); - } else { - mResponse->set_status(::aurum::RspStatus::ERROR); + std::shared_ptr mDevice = UiDevice::getInstance(); + +#ifdef MQTT_ENABLED + if (mDevice->getExternalAppLaunched()) + { + mDevice->RequestScreenAnalyze(); + + LOGI("Search Object start"); + + auto found = mDevice->getSAWatcher()->findSaObject(selector); + if (found != nullptr) { + SaObject *obj = found.get(); + ::aurum::Element *elm = mResponse->mutable_element(); + elm->set_elementid(obj->getId()); + + ::aurum::Rect *rect = elm->mutable_geometry(); + const Rect &size = obj->getScreenBoundingBox(); + rect->set_x(size.mTopLeft.x); + rect->set_y(size.mTopLeft.y); + rect->set_width(size.width()); + rect->set_height(size.height()); + + elm->set_widget_type(obj->getType()); + + elm->set_text(obj->getOcrText()); + + elm->set_isclickable(obj->isClickable()); + elm->set_isfocused(obj->isFocused()); + elm->set_isfocusable(obj->isFocusable()); + elm->set_isactive(obj->isActive()); + elm->set_isshowing(true); + elm->set_isvisible(true); + + mResponse->set_status(::aurum::RspStatus::OK); + } + else + mResponse->set_status(::aurum::RspStatus::ERROR); + } + else +#endif + { + if (mDevice->getWithScreenAnalyzer()) + { + LOGI("Find object with SA information"); + mDevice->RequestScreenAnalyze(); + } + + auto found = searchableObj->findObject(selector); + + if (found != nullptr) { + UiObject *obj = found.get(); + obj->refresh(); + if (mObjMap->getElement(obj->getId()) == nullptr) + mObjMap->addElement(std::move(found)); + + LOGI("found object : %p elementId:%s", obj, obj->getId().c_str()); + + ::aurum::Element *elm = mResponse->mutable_element(); + elm->set_elementid(obj->getId()); + elm->set_package(obj->getApplicationPackage()); + + ::aurum::Rect *rect = elm->mutable_geometry(); + const Rect &size = obj->getScreenBoundingBox(); + rect->set_x(size.mTopLeft.x); + rect->set_y(size.mTopLeft.y); + rect->set_width(size.width()); + rect->set_height(size.height()); + + ::aurum::Rect *windowRect = elm->mutable_window_relative_geometry(); + const Rect &windowRelativeSize = obj->getWindowBoundingBox(); + windowRect->set_x(windowRelativeSize.mTopLeft.x); + windowRect->set_y(windowRelativeSize.mTopLeft.y); + windowRect->set_width(windowRelativeSize.width()); + windowRect->set_height(windowRelativeSize.height()); + + elm->set_widget_type(obj->getType()); + elm->set_widget_style(obj->getElementStyle()); + + elm->set_text(obj->getText()); + elm->set_xpath(obj->getXPath()); + elm->set_ocrtext(obj->getOcrText()); + elm->set_automationid(obj->getAutomationId()); + elm->set_package(obj->getApplicationPackage()); + elm->set_role(obj->getRole()); + + elm->set_ischecked(obj->isChecked()); + elm->set_ischeckable(obj->isCheckable()); + elm->set_isclickable(obj->isClickable()); + elm->set_isenabled(obj->isEnabled()); + elm->set_isfocused(obj->isFocused()); + elm->set_isfocusable(obj->isFocusable()); + elm->set_isscrollable(obj->isScrollable()); + elm->set_isselected(obj->isSelected()); + elm->set_isshowing(obj->isShowing()); + elm->set_isactive(obj->isActive()); + elm->set_isvisible(obj->isVisible()); + elm->set_isselectable(obj->isSelectable()); + + elm->set_minvalue(obj->getMinValue()); + elm->set_maxvalue(obj->getMaxValue()); + elm->set_value(obj->getValue()); + elm->set_increment(obj->getIncrement()); + + mResponse->set_status(::aurum::RspStatus::OK); + } else { + mResponse->set_status(::aurum::RspStatus::ERROR); + } } return grpc::Status::OK; diff --git a/org.tizen.aurum-bootstrap/src/Commands/FindElementsCommand.cc b/org.tizen.aurum-bootstrap/src/Commands/FindElementsCommand.cc index 2efc5c7..bc2119c 100644 --- a/org.tizen.aurum-bootstrap/src/Commands/FindElementsCommand.cc +++ b/org.tizen.aurum-bootstrap/src/Commands/FindElementsCommand.cc @@ -22,6 +22,9 @@ #include "UiSelector.h" #include "Sel.h" #include "ISearchable.h" +#ifdef MQTT_ENABLED +#include "SaObject.h" +#endif FindElementsCommand::FindElementsCommand(const ::aurum::ReqFindElements *request, ::aurum::RspFindElements *response) @@ -65,6 +68,7 @@ std::vector> FindElementsCommand::getSelectors(void) if(mRequest->_packagename_case()) sel->pkg(mRequest->packagename()); if(mRequest->_textpartialmatch_case()) sel->textPartialMatch(mRequest->textpartialmatch()); if(mRequest->_xpath_case()) sel->xpath(mRequest->xpath()); + if(mRequest->_ocrtext_case()) sel->ocrText(mRequest->ocrtext()); return std::vector>{sel}; } @@ -74,71 +78,125 @@ std::vector> FindElementsCommand::getSelectors(void) LOGI("findElements --------------- "); auto searchableObj = getSearchableTop(); auto selectors = getSelectors(); + std::shared_ptr mDevice = UiDevice::getInstance(); - std::vector> founds = {}; +#ifdef MQTT_ENABLED + if (mDevice->getExternalAppLaunched()) + { + mDevice->RequestScreenAnalyze(); - for ( auto &sel : selectors ) { - auto ret = searchableObj->findObjects(sel); - std::move(std::begin(ret), std::end(ret), std::back_inserter(founds)); + std::vector> founds = {}; + + for ( auto &sel : selectors ) { + auto ret = mDevice->getSAWatcher()->findSaObjects(sel); + std::move(std::begin(ret), std::end(ret), std::back_inserter(founds)); + } + if (founds.size() > 0) { + for (auto& found : founds) { + SaObject *obj = found.get(); + ::aurum::Element *elm = mResponse->add_elements(); + elm->set_elementid(obj->getId()); + + ::aurum::Rect *rect = elm->mutable_geometry(); + const Rect &size = obj->getScreenBoundingBox(); + rect->set_x(size.mTopLeft.x); + rect->set_y(size.mTopLeft.y); + rect->set_width(size.width()); + rect->set_height(size.height()); + + elm->set_widget_type(obj->getType()); + + elm->set_text(obj->getOcrText()); + elm->set_toolkit("External"); + + elm->set_isclickable(obj->isClickable()); + elm->set_isfocused(obj->isFocused()); + elm->set_isfocusable(obj->isFocusable()); + elm->set_isactive(obj->isActive()); + elm->set_isshowing(true); + elm->set_isvisible(true); + } + mResponse->set_status(::aurum::RspStatus::OK); + } + else { + mResponse->set_status(::aurum::RspStatus::ERROR); + } } + else +#endif + { + if (mDevice->getWithScreenAnalyzer()) + { + LOGI("Find object with SA information"); + mDevice->RequestScreenAnalyze(); + } + + std::vector> founds = {}; + + for ( auto &sel : selectors ) { + auto ret = searchableObj->findObjects(sel); + std::move(std::begin(ret), std::end(ret), std::back_inserter(founds)); + } - if (founds.size() > 0) { - for (auto& found : founds) { - UiObject *obj = found.get(); - obj->refresh(); - if (mObjMap->getElement(obj->getId()) == nullptr) - mObjMap->addElement(std::move(found)); - - LOGI("found object : %p elementId:%s", obj, obj->getId().c_str()); - - ::aurum::Element *elm = mResponse->add_elements(); - elm->set_elementid(obj->getId()); - elm->set_package(obj->getApplicationPackage()); - - ::aurum::Rect *rect = elm->mutable_geometry(); - const Rect &size = obj->getScreenBoundingBox(); - rect->set_x(size.mTopLeft.x); - rect->set_y(size.mTopLeft.y); - rect->set_width(size.width()); - rect->set_height(size.height()); - - ::aurum::Rect *windowRect = elm->mutable_window_relative_geometry(); - const Rect &windowRelativeSize = obj->getWindowBoundingBox(); - windowRect->set_x(windowRelativeSize.mTopLeft.x); - windowRect->set_y(windowRelativeSize.mTopLeft.y); - windowRect->set_width(windowRelativeSize.width()); - windowRect->set_height(windowRelativeSize.height()); - - elm->set_widget_type(obj->getElementType()); - elm->set_widget_style(obj->getElementStyle()); - - elm->set_text(obj->getText()); - elm->set_xpath(obj->getXPath()); - elm->set_automationid(obj->getAutomationId()); - elm->set_package(obj->getApplicationPackage()); - elm->set_role(obj->getRole()); - - elm->set_ischecked(obj->isChecked()); - elm->set_ischeckable(obj->isCheckable()); - elm->set_isclickable(obj->isClickable()); - elm->set_isenabled(obj->isEnabled()); - elm->set_isfocused(obj->isFocused()); - elm->set_isfocusable(obj->isFocusable()); - elm->set_isscrollable(obj->isScrollable()); - elm->set_isselected(obj->isSelected()); - elm->set_isshowing(obj->isShowing()); - elm->set_isactive(obj->isActive()); - elm->set_isvisible(obj->isVisible()); - elm->set_isselectable(obj->isSelectable()); - - elm->set_minvalue(obj->getMinValue()); - elm->set_maxvalue(obj->getMaxValue()); - elm->set_value(obj->getValue()); - elm->set_increment(obj->getIncrement()); + if (founds.size() > 0) { + for (auto& found : founds) { + UiObject *obj = found.get(); + obj->refresh(); + if (mObjMap->getElement(obj->getId()) == nullptr) + mObjMap->addElement(std::move(found)); + + LOGI("found object : %p elementId:%s", obj, obj->getId().c_str()); + + ::aurum::Element *elm = mResponse->add_elements(); + elm->set_elementid(obj->getId()); + elm->set_package(obj->getApplicationPackage()); + + ::aurum::Rect *rect = elm->mutable_geometry(); + const Rect &size = obj->getScreenBoundingBox(); + rect->set_x(size.mTopLeft.x); + rect->set_y(size.mTopLeft.y); + rect->set_width(size.width()); + rect->set_height(size.height()); + + ::aurum::Rect *windowRect = elm->mutable_window_relative_geometry(); + const Rect &windowRelativeSize = obj->getWindowBoundingBox(); + windowRect->set_x(windowRelativeSize.mTopLeft.x); + windowRect->set_y(windowRelativeSize.mTopLeft.y); + windowRect->set_width(windowRelativeSize.width()); + windowRect->set_height(windowRelativeSize.height()); + + elm->set_widget_type(obj->getType()); + elm->set_widget_style(obj->getElementStyle()); + + elm->set_text(obj->getText()); + elm->set_xpath(obj->getXPath()); + elm->set_ocrtext(obj->getOcrText()); + elm->set_automationid(obj->getAutomationId()); + elm->set_package(obj->getApplicationPackage()); + elm->set_role(obj->getRole()); + + elm->set_ischecked(obj->isChecked()); + elm->set_ischeckable(obj->isCheckable()); + elm->set_isclickable(obj->isClickable()); + elm->set_isenabled(obj->isEnabled()); + elm->set_isfocused(obj->isFocused()); + elm->set_isfocusable(obj->isFocusable()); + elm->set_isscrollable(obj->isScrollable()); + elm->set_isselected(obj->isSelected()); + elm->set_isshowing(obj->isShowing()); + elm->set_isactive(obj->isActive()); + elm->set_isvisible(obj->isVisible()); + elm->set_isselectable(obj->isSelectable()); + + elm->set_minvalue(obj->getMinValue()); + elm->set_maxvalue(obj->getMaxValue()); + elm->set_value(obj->getValue()); + elm->set_increment(obj->getIncrement()); + } + mResponse->set_status(::aurum::RspStatus::OK); + } else { + mResponse->set_status(::aurum::RspStatus::ERROR); } - mResponse->set_status(::aurum::RspStatus::OK); - } else { - mResponse->set_status(::aurum::RspStatus::ERROR); } return grpc::Status::OK; diff --git a/org.tizen.aurum-bootstrap/src/Commands/GetActiveAppToolkitNameCommand.cc b/org.tizen.aurum-bootstrap/src/Commands/GetActiveAppToolkitNameCommand.cc new file mode 100644 index 0000000..cb9d869 --- /dev/null +++ b/org.tizen.aurum-bootstrap/src/Commands/GetActiveAppToolkitNameCommand.cc @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2021 Samsung Electronics Co., Ltd All Rights Reserved + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "bootstrap.h" +#include "GetActiveAppToolkitNameCommand.h" +#include "UiDevice.h" + +GetActiveAppToolkitNameCommand::GetActiveAppToolkitNameCommand(const ::aurum::ReqGetActiveAppToolkitName *request, + ::aurum::RspGetActiveAppToolkitName *response) + : mRequest{request}, mResponse{response} +{ +} + +::grpc::Status GetActiveAppToolkitNameCommand::execute() +{ + LOGI("GetActiveAppToolkitName --------------- "); + + return grpc::Status::OK; +} diff --git a/org.tizen.aurum-bootstrap/src/Commands/GetSizeCommand.cc b/org.tizen.aurum-bootstrap/src/Commands/GetSizeCommand.cc index dfada9c..534e5a1 100644 --- a/org.tizen.aurum-bootstrap/src/Commands/GetSizeCommand.cc +++ b/org.tizen.aurum-bootstrap/src/Commands/GetSizeCommand.cc @@ -18,6 +18,14 @@ #include "bootstrap.h" #include "GetSizeCommand.h" #include "UiObject.h" +#include "UiDevice.h" +#include "UiSelector.h" +#include "Sel.h" +#include "ISearchable.h" +#ifdef MQTT_ENABLED +#include "SaObject.h" +#endif + GetSizeCommand::GetSizeCommand(const ::aurum::ReqGetSize *request, ::aurum::RspGetSize *response) @@ -29,27 +37,60 @@ GetSizeCommand::GetSizeCommand(const ::aurum::ReqGetSize *request, { LOGI("GetSize --------------- "); - ::aurum::ReqGetSize_CoordType type = mRequest->type(); - ObjectMapper *mObjMap = ObjectMapper::getInstance(); - std::shared_ptr obj = mObjMap->getElement(mRequest->elementid()); - if (obj) { - obj->updateExtents(); - ::aurum::Rect *rect = mResponse->mutable_size(); - if (type == ::aurum::ReqGetSize_CoordType::ReqGetSize_CoordType_SCREEN) { - const Rect &size = obj->getScreenBoundingBox(); - rect->set_x(size.mTopLeft.x); - rect->set_y(size.mTopLeft.y); - rect->set_width(size.width()); - rect->set_height(size.height()); + std::shared_ptr mDevice = UiDevice::getInstance(); + +#ifdef MQTT_ENABLED + if (mDevice->getExternalAppLaunched()) + { + mDevice->RequestScreenAnalyze(); + + std::vector> founds = {}; + + auto tempSel = std::make_shared(); + tempSel->id(mRequest->elementid()); + auto selectors = std::vector>{tempSel}; + + for ( auto &sel : selectors ) { + auto ret = mDevice->getSAWatcher()->findSaObjects(sel); + std::move(std::begin(ret), std::end(ret), std::back_inserter(founds)); + } + if (founds.size() > 0) { + for (auto& found : founds) { + SaObject *obj = found.get(); + ::aurum::Rect *rect = mResponse->mutable_size(); + const Rect &size = obj->getScreenBoundingBox(); + rect->set_x(size.mTopLeft.x); + rect->set_y(size.mTopLeft.y); + rect->set_width(size.width()); + rect->set_height(size.height()); + } } - else { - const Rect &windowRelativeSize = obj->getWindowBoundingBox(); - rect->set_x(windowRelativeSize.mTopLeft.x); - rect->set_y(windowRelativeSize.mTopLeft.y); - rect->set_width(windowRelativeSize.width()); - rect->set_height(windowRelativeSize.height()); + } + else +#endif + { + ::aurum::ReqGetSize_CoordType type = mRequest->type(); + ObjectMapper *mObjMap = ObjectMapper::getInstance(); + std::shared_ptr obj = mObjMap->getElement(mRequest->elementid()); + if (obj) { + obj->updateExtents(); + ::aurum::Rect *rect = mResponse->mutable_size(); + if (type == ::aurum::ReqGetSize_CoordType::ReqGetSize_CoordType_SCREEN) { + const Rect &size = obj->getScreenBoundingBox(); + rect->set_x(size.mTopLeft.x); + rect->set_y(size.mTopLeft.y); + rect->set_width(size.width()); + rect->set_height(size.height()); + } + else { + const Rect &windowRelativeSize = obj->getWindowBoundingBox(); + rect->set_x(windowRelativeSize.mTopLeft.x); + rect->set_y(windowRelativeSize.mTopLeft.y); + rect->set_width(windowRelativeSize.width()); + rect->set_height(windowRelativeSize.height()); + } } } return grpc::Status::OK; -} \ No newline at end of file +} diff --git a/packaging/aurum.spec b/packaging/aurum.spec index ce0c060..0928e3a 100644 --- a/packaging/aurum.spec +++ b/packaging/aurum.spec @@ -31,6 +31,11 @@ BuildRequires: pkgconfig(capi-system-system-settings) BuildRequires: pkgconfig(capi-base-utils-i18n) BuildRequires: pkgconfig(vconf) +%if "%{mqtt}" == "1" +BuildRequires: pkgconfig(libmosquitto) +BuildRequires: pkgconfig(jsoncpp) +%endif + %if 0%{?gendoc:1} BuildRequires: doxygen %endif @@ -82,19 +87,16 @@ Group: Graphics & UI Framework/Testing Ui Automation Library Aurum gcov objects %endif - %prep %setup -q cp %{SOURCE1001} . - %if "%{asan}" == "1" %restore_fcommon %else export LDFLAGS+="-Wl,-z,noexecstack" %endif - %if 0%{?gcov:1} export CFLAGS+=" -fprofile-arcs -ftest-coverage " export CXXFLAGS+=" -fprofile-arcs -ftest-coverage " @@ -110,12 +112,20 @@ export LDFLAGS+=" -lgcov" %else %define TIZEN_GEN_DOC false %endif + +%if "%{mqtt}" == "1" +%define MQTT_ENABLED true +%else +%define MQTT_ENABLED false +%endif + meson \ --prefix /usr \ --libdir %{_libdir} \ -Dcpp_std=c++17 \ -Dtizen=true \ -Denable_documentation=%{TIZEN_GEN_DOC} \ + -Dmqtt_enabled=%{MQTT_ENABLED} \ -Dtizen_gcov=%{TIZEN_GCOV} \ -Dtzapp_path=%{TZ_SYS_RO_APP} \ -Dtzpackage_path=%{TZ_SYS_RO_PACKAGES} \ diff --git a/protocol/aurum.proto b/protocol/aurum.proto index 6b7d454..51600f5 100644 --- a/protocol/aurum.proto +++ b/protocol/aurum.proto @@ -33,6 +33,8 @@ service Bootstrap { rpc actionAndWaitEvent(ReqActionAndWaitEvent) returns (RspActionAndWaitEvent) {} rpc setFocus(ReqSetFocus) returns (RspSetFocus) {} rpc findElements(ReqFindElements) returns (RspFindElements) {} + rpc getActiveAppToolkitName(ReqGetActiveAppToolkitName) returns (RspGetActiveAppToolkitName) {} + rpc enableScreenAnalyzer(ReqEnableScreenAnalyzer) returns (RspEnableScreenAnalyzer) {} } // ------------------------------------ // @@ -62,27 +64,29 @@ message Element { string text = 7; string xpath = 8; - string automationId = 9; - string package = 10; - string role = 11; - - bool isChecked = 12; - bool isCheckable = 13; - bool isClickable = 14; - bool isEnabled = 15; - bool isFocused = 16; - bool isFocusable = 17; - bool isScrollable = 18; - bool isSelected = 19; - bool isShowing = 20; - bool isActive = 21; - bool isVisible = 22; - bool isSelectable = 23; - - double minValue = 24; - double maxValue = 25; - double value = 26; - double increment = 27; + string ocrText = 9; + string automationId = 10; + string package = 11; + string role = 12; + string toolkit = 13; + + bool isChecked = 14; + bool isCheckable = 15; + bool isClickable = 16; + bool isEnabled = 17; + bool isFocused = 18; + bool isFocusable = 19; + bool isScrollable = 20; + bool isSelected = 21; + bool isShowing = 22; + bool isActive = 23; + bool isVisible = 24; + bool isSelectable = 25; + + double minValue = 26; + double maxValue = 27; + double value = 28; + double increment = 29; } message Point { @@ -184,7 +188,11 @@ message ReqFindElement { string xpath = 20; } - repeated ReqFindElement children = 21; + oneof _ocrtext { + string ocrText = 21; + } + + repeated ReqFindElement children = 22; } message RspFindElement { @@ -272,7 +280,10 @@ message ReqFindElements { string xpath = 20; } - repeated ReqFindElements children = 21; + oneof _ocrtext { + string ocrText = 21; + } + repeated ReqFindElements children = 22; } message RspFindElements { @@ -564,3 +575,21 @@ message ReqSetFocus { message RspSetFocus { RspStatus status = 1; } + +message ReqGetActiveAppToolkitName { +} + +message RspGetActiveAppToolkitName { + RspStatus status = 1; + string toolkitName = 2; +} + +message ReqEnableScreenAnalyzer { + bool enable = 1; + string serverIp = 2; +} + +message RspEnableScreenAnalyzer { + RspStatus status = 1; +} + diff --git a/tests/Test_UiObject.cc b/tests/Test_UiObject.cc index 9f20ca8..8b9742f 100644 --- a/tests/Test_UiObject.cc +++ b/tests/Test_UiObject.cc @@ -189,11 +189,11 @@ TEST_F(AurumTestUiObject, getAutomationId) ASSERT_EQ(parent->getAutomationId(), ""); } -TEST_F(AurumTestUiObject, getElementType_P1) +TEST_F(AurumTestUiObject, getType_P1) { auto obj = UiDevice::getInstance(); auto parent = obj->findObject(Sel::text("test2")); - ASSERT_EQ(parent->getElementType(), "type"); + ASSERT_EQ(parent->getType(), "type"); } TEST_F(AurumTestUiObject, getElementStyle_P1) diff --git a/tests/Test_UiSelector.cc b/tests/Test_UiSelector.cc index bdf6861..def5a33 100644 --- a/tests/Test_UiSelector.cc +++ b/tests/Test_UiSelector.cc @@ -185,7 +185,7 @@ TEST_F(AurumTestUiSelector, Selector_Advanced_P1) sel->text("win1"); auto found = UiDevice::getInstance()->findObject(sel); - ASSERT_EQ(found->getElementType(), "Elm_Win"); + ASSERT_EQ(found->getType(), "Elm_Win"); auto children = found->getChildren(); ASSERT_EQ(children.size(), 4); @@ -205,7 +205,7 @@ TEST_F(AurumTestUiSelector, Selector_Advanced_P2) sel->text("win1"); auto found = UiDevice::getInstance()->findObject(sel); - ASSERT_EQ(found->getElementType(), "Elm_Win"); + ASSERT_EQ(found->getType(), "Elm_Win"); auto children = found->getChildren(); ASSERT_EQ(children.size(), 4); @@ -282,7 +282,7 @@ TEST_F(AurumTestUiSelector, Selector_Advanced_N1) sel->text("win1"); auto found = UiDevice::getInstance()->findObject(sel); - ASSERT_EQ(found->getElementType(), "Elm_Win"); + ASSERT_EQ(found->getType(), "Elm_Win"); auto children = found->getChildren(); ASSERT_EQ(children.size(), 4); -- 2.34.1